{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ab4d0442",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e9d6261b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import (\n",
    "    AutoTokenizer,\n",
    "    T5ForConditionalGeneration,\n",
    "    AutoConfig,\n",
    ")\n",
    "from pytorch_lightning import LightningModule\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d426b62d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('./out-large-1.1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "94be4a10",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Module(LightningModule):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        config = AutoConfig.from_pretrained(\n",
    "            './out-large-1.1'\n",
    "        )\n",
    "        self.model = T5ForConditionalGeneration.from_pretrained(\n",
    "            './out-large-1.1',\n",
    "            config=config,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e4b5d292",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 44G\r\n",
      "-rwxrwxrwx 1 ubuntu ubuntu 8.8G Oct 26 02:34 'model-epoch=00-step=1000.ckpt'\r\n",
      "-rwxrwxrwx 1 ubuntu ubuntu 8.8G Oct 26 03:59 'model-epoch=00-step=2000.ckpt'\r\n",
      "-rw-r--r-- 1 ubuntu ubuntu 8.8G Oct 26 04:52 'model-epoch=00-step=2400.ckpt'\r\n",
      "-rw-r--r-- 1 ubuntu ubuntu 8.8G Oct 26 05:22 'model-epoch=00-step=2800.ckpt'\r\n",
      "-rw-r--r-- 1 ubuntu ubuntu 8.8G Oct 26 05:52 'model-epoch=00-step=3200.ckpt'\r\n"
     ]
    }
   ],
   "source": [
    "!ls -lh logs/large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "74be228f",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Module()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fc74cda8",
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = model.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cd9d6272",
   "metadata": {},
   "outputs": [],
   "source": [
    "old_weights = torch.load('logs/large/model-epoch=00-step=3200.ckpt',\n",
    "                             map_location=torch.device('cpu'))['state_dict'].items()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "652becba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model.shared.weight model.shared.weight\n",
      "model.encoder.embed_tokens.weight model.encoder.embed_tokens.weight\n",
      "model.encoder.block.0.layer.0.SelfAttention.q.weight model.encoder.block.0.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.0.layer.0.SelfAttention.k.weight model.encoder.block.0.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.0.layer.0.SelfAttention.v.weight model.encoder.block.0.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.0.layer.0.SelfAttention.o.weight model.encoder.block.0.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight model.encoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight\n",
      "model.encoder.block.0.layer.0.layer_norm.weight model.encoder.block.0.layer.0.layer_norm.weight\n",
      "model.encoder.block.0.layer.1.DenseReluDense.wi_0.weight model.encoder.block.0.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.0.layer.1.DenseReluDense.wi_1.weight model.encoder.block.0.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.0.layer.1.DenseReluDense.wo.weight model.encoder.block.0.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.0.layer.1.layer_norm.weight model.encoder.block.0.layer.1.layer_norm.weight\n",
      "model.encoder.block.1.layer.0.SelfAttention.q.weight model.encoder.block.1.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.1.layer.0.SelfAttention.k.weight model.encoder.block.1.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.1.layer.0.SelfAttention.v.weight model.encoder.block.1.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.1.layer.0.SelfAttention.o.weight model.encoder.block.1.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.1.layer.0.layer_norm.weight model.encoder.block.1.layer.0.layer_norm.weight\n",
      "model.encoder.block.1.layer.1.DenseReluDense.wi_0.weight model.encoder.block.1.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.1.layer.1.DenseReluDense.wi_1.weight model.encoder.block.1.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.1.layer.1.DenseReluDense.wo.weight model.encoder.block.1.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.1.layer.1.layer_norm.weight model.encoder.block.1.layer.1.layer_norm.weight\n",
      "model.encoder.block.2.layer.0.SelfAttention.q.weight model.encoder.block.2.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.2.layer.0.SelfAttention.k.weight model.encoder.block.2.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.2.layer.0.SelfAttention.v.weight model.encoder.block.2.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.2.layer.0.SelfAttention.o.weight model.encoder.block.2.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.2.layer.0.layer_norm.weight model.encoder.block.2.layer.0.layer_norm.weight\n",
      "model.encoder.block.2.layer.1.DenseReluDense.wi_0.weight model.encoder.block.2.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.2.layer.1.DenseReluDense.wi_1.weight model.encoder.block.2.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.2.layer.1.DenseReluDense.wo.weight model.encoder.block.2.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.2.layer.1.layer_norm.weight model.encoder.block.2.layer.1.layer_norm.weight\n",
      "model.encoder.block.3.layer.0.SelfAttention.q.weight model.encoder.block.3.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.3.layer.0.SelfAttention.k.weight model.encoder.block.3.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.3.layer.0.SelfAttention.v.weight model.encoder.block.3.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.3.layer.0.SelfAttention.o.weight model.encoder.block.3.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.3.layer.0.layer_norm.weight model.encoder.block.3.layer.0.layer_norm.weight\n",
      "model.encoder.block.3.layer.1.DenseReluDense.wi_0.weight model.encoder.block.3.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.3.layer.1.DenseReluDense.wi_1.weight model.encoder.block.3.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.3.layer.1.DenseReluDense.wo.weight model.encoder.block.3.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.3.layer.1.layer_norm.weight model.encoder.block.3.layer.1.layer_norm.weight\n",
      "model.encoder.block.4.layer.0.SelfAttention.q.weight model.encoder.block.4.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.4.layer.0.SelfAttention.k.weight model.encoder.block.4.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.4.layer.0.SelfAttention.v.weight model.encoder.block.4.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.4.layer.0.SelfAttention.o.weight model.encoder.block.4.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.4.layer.0.layer_norm.weight model.encoder.block.4.layer.0.layer_norm.weight\n",
      "model.encoder.block.4.layer.1.DenseReluDense.wi_0.weight model.encoder.block.4.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.4.layer.1.DenseReluDense.wi_1.weight model.encoder.block.4.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.4.layer.1.DenseReluDense.wo.weight model.encoder.block.4.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.4.layer.1.layer_norm.weight model.encoder.block.4.layer.1.layer_norm.weight\n",
      "model.encoder.block.5.layer.0.SelfAttention.q.weight model.encoder.block.5.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.5.layer.0.SelfAttention.k.weight model.encoder.block.5.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.5.layer.0.SelfAttention.v.weight model.encoder.block.5.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.5.layer.0.SelfAttention.o.weight model.encoder.block.5.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.5.layer.0.layer_norm.weight model.encoder.block.5.layer.0.layer_norm.weight\n",
      "model.encoder.block.5.layer.1.DenseReluDense.wi_0.weight model.encoder.block.5.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.5.layer.1.DenseReluDense.wi_1.weight model.encoder.block.5.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.5.layer.1.DenseReluDense.wo.weight model.encoder.block.5.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.5.layer.1.layer_norm.weight model.encoder.block.5.layer.1.layer_norm.weight\n",
      "model.encoder.block.6.layer.0.SelfAttention.q.weight model.encoder.block.6.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.6.layer.0.SelfAttention.k.weight model.encoder.block.6.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.6.layer.0.SelfAttention.v.weight model.encoder.block.6.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.6.layer.0.SelfAttention.o.weight model.encoder.block.6.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.6.layer.0.layer_norm.weight model.encoder.block.6.layer.0.layer_norm.weight\n",
      "model.encoder.block.6.layer.1.DenseReluDense.wi_0.weight model.encoder.block.6.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.6.layer.1.DenseReluDense.wi_1.weight model.encoder.block.6.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.6.layer.1.DenseReluDense.wo.weight model.encoder.block.6.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.6.layer.1.layer_norm.weight model.encoder.block.6.layer.1.layer_norm.weight\n",
      "model.encoder.block.7.layer.0.SelfAttention.q.weight model.encoder.block.7.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.7.layer.0.SelfAttention.k.weight model.encoder.block.7.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.7.layer.0.SelfAttention.v.weight model.encoder.block.7.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.7.layer.0.SelfAttention.o.weight model.encoder.block.7.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.7.layer.0.layer_norm.weight model.encoder.block.7.layer.0.layer_norm.weight\n",
      "model.encoder.block.7.layer.1.DenseReluDense.wi_0.weight model.encoder.block.7.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.7.layer.1.DenseReluDense.wi_1.weight model.encoder.block.7.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.7.layer.1.DenseReluDense.wo.weight model.encoder.block.7.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.7.layer.1.layer_norm.weight model.encoder.block.7.layer.1.layer_norm.weight\n",
      "model.encoder.block.8.layer.0.SelfAttention.q.weight model.encoder.block.8.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.8.layer.0.SelfAttention.k.weight model.encoder.block.8.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.8.layer.0.SelfAttention.v.weight model.encoder.block.8.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.8.layer.0.SelfAttention.o.weight model.encoder.block.8.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.8.layer.0.layer_norm.weight model.encoder.block.8.layer.0.layer_norm.weight\n",
      "model.encoder.block.8.layer.1.DenseReluDense.wi_0.weight model.encoder.block.8.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.8.layer.1.DenseReluDense.wi_1.weight model.encoder.block.8.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.8.layer.1.DenseReluDense.wo.weight model.encoder.block.8.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.8.layer.1.layer_norm.weight model.encoder.block.8.layer.1.layer_norm.weight\n",
      "model.encoder.block.9.layer.0.SelfAttention.q.weight model.encoder.block.9.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.9.layer.0.SelfAttention.k.weight model.encoder.block.9.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.9.layer.0.SelfAttention.v.weight model.encoder.block.9.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.9.layer.0.SelfAttention.o.weight model.encoder.block.9.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.9.layer.0.layer_norm.weight model.encoder.block.9.layer.0.layer_norm.weight\n",
      "model.encoder.block.9.layer.1.DenseReluDense.wi_0.weight model.encoder.block.9.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.9.layer.1.DenseReluDense.wi_1.weight model.encoder.block.9.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.9.layer.1.DenseReluDense.wo.weight model.encoder.block.9.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.9.layer.1.layer_norm.weight model.encoder.block.9.layer.1.layer_norm.weight\n",
      "model.encoder.block.10.layer.0.SelfAttention.q.weight model.encoder.block.10.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.10.layer.0.SelfAttention.k.weight model.encoder.block.10.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.10.layer.0.SelfAttention.v.weight model.encoder.block.10.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.10.layer.0.SelfAttention.o.weight model.encoder.block.10.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.10.layer.0.layer_norm.weight model.encoder.block.10.layer.0.layer_norm.weight\n",
      "model.encoder.block.10.layer.1.DenseReluDense.wi_0.weight model.encoder.block.10.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.10.layer.1.DenseReluDense.wi_1.weight model.encoder.block.10.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.10.layer.1.DenseReluDense.wo.weight model.encoder.block.10.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.10.layer.1.layer_norm.weight model.encoder.block.10.layer.1.layer_norm.weight\n",
      "model.encoder.block.11.layer.0.SelfAttention.q.weight model.encoder.block.11.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.11.layer.0.SelfAttention.k.weight model.encoder.block.11.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.11.layer.0.SelfAttention.v.weight model.encoder.block.11.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.11.layer.0.SelfAttention.o.weight model.encoder.block.11.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.11.layer.0.layer_norm.weight model.encoder.block.11.layer.0.layer_norm.weight\n",
      "model.encoder.block.11.layer.1.DenseReluDense.wi_0.weight model.encoder.block.11.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.11.layer.1.DenseReluDense.wi_1.weight model.encoder.block.11.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.11.layer.1.DenseReluDense.wo.weight model.encoder.block.11.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.11.layer.1.layer_norm.weight model.encoder.block.11.layer.1.layer_norm.weight\n",
      "model.encoder.block.12.layer.0.SelfAttention.q.weight model.encoder.block.12.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.12.layer.0.SelfAttention.k.weight model.encoder.block.12.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.12.layer.0.SelfAttention.v.weight model.encoder.block.12.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.12.layer.0.SelfAttention.o.weight model.encoder.block.12.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.12.layer.0.layer_norm.weight model.encoder.block.12.layer.0.layer_norm.weight\n",
      "model.encoder.block.12.layer.1.DenseReluDense.wi_0.weight model.encoder.block.12.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.12.layer.1.DenseReluDense.wi_1.weight model.encoder.block.12.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.12.layer.1.DenseReluDense.wo.weight model.encoder.block.12.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.12.layer.1.layer_norm.weight model.encoder.block.12.layer.1.layer_norm.weight\n",
      "model.encoder.block.13.layer.0.SelfAttention.q.weight model.encoder.block.13.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.13.layer.0.SelfAttention.k.weight model.encoder.block.13.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.13.layer.0.SelfAttention.v.weight model.encoder.block.13.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.13.layer.0.SelfAttention.o.weight model.encoder.block.13.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.13.layer.0.layer_norm.weight model.encoder.block.13.layer.0.layer_norm.weight\n",
      "model.encoder.block.13.layer.1.DenseReluDense.wi_0.weight model.encoder.block.13.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.13.layer.1.DenseReluDense.wi_1.weight model.encoder.block.13.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.13.layer.1.DenseReluDense.wo.weight model.encoder.block.13.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.13.layer.1.layer_norm.weight model.encoder.block.13.layer.1.layer_norm.weight\n",
      "model.encoder.block.14.layer.0.SelfAttention.q.weight model.encoder.block.14.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.14.layer.0.SelfAttention.k.weight model.encoder.block.14.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.14.layer.0.SelfAttention.v.weight model.encoder.block.14.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.14.layer.0.SelfAttention.o.weight model.encoder.block.14.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.14.layer.0.layer_norm.weight model.encoder.block.14.layer.0.layer_norm.weight\n",
      "model.encoder.block.14.layer.1.DenseReluDense.wi_0.weight model.encoder.block.14.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.14.layer.1.DenseReluDense.wi_1.weight model.encoder.block.14.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.14.layer.1.DenseReluDense.wo.weight model.encoder.block.14.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.14.layer.1.layer_norm.weight model.encoder.block.14.layer.1.layer_norm.weight\n",
      "model.encoder.block.15.layer.0.SelfAttention.q.weight model.encoder.block.15.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.15.layer.0.SelfAttention.k.weight model.encoder.block.15.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.15.layer.0.SelfAttention.v.weight model.encoder.block.15.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.15.layer.0.SelfAttention.o.weight model.encoder.block.15.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.15.layer.0.layer_norm.weight model.encoder.block.15.layer.0.layer_norm.weight\n",
      "model.encoder.block.15.layer.1.DenseReluDense.wi_0.weight model.encoder.block.15.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.15.layer.1.DenseReluDense.wi_1.weight model.encoder.block.15.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.15.layer.1.DenseReluDense.wo.weight model.encoder.block.15.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.15.layer.1.layer_norm.weight model.encoder.block.15.layer.1.layer_norm.weight\n",
      "model.encoder.block.16.layer.0.SelfAttention.q.weight model.encoder.block.16.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.16.layer.0.SelfAttention.k.weight model.encoder.block.16.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.16.layer.0.SelfAttention.v.weight model.encoder.block.16.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.16.layer.0.SelfAttention.o.weight model.encoder.block.16.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.16.layer.0.layer_norm.weight model.encoder.block.16.layer.0.layer_norm.weight\n",
      "model.encoder.block.16.layer.1.DenseReluDense.wi_0.weight model.encoder.block.16.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.16.layer.1.DenseReluDense.wi_1.weight model.encoder.block.16.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.16.layer.1.DenseReluDense.wo.weight model.encoder.block.16.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.16.layer.1.layer_norm.weight model.encoder.block.16.layer.1.layer_norm.weight\n",
      "model.encoder.block.17.layer.0.SelfAttention.q.weight model.encoder.block.17.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.17.layer.0.SelfAttention.k.weight model.encoder.block.17.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.17.layer.0.SelfAttention.v.weight model.encoder.block.17.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.17.layer.0.SelfAttention.o.weight model.encoder.block.17.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.17.layer.0.layer_norm.weight model.encoder.block.17.layer.0.layer_norm.weight\n",
      "model.encoder.block.17.layer.1.DenseReluDense.wi_0.weight model.encoder.block.17.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.17.layer.1.DenseReluDense.wi_1.weight model.encoder.block.17.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.17.layer.1.DenseReluDense.wo.weight model.encoder.block.17.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.17.layer.1.layer_norm.weight model.encoder.block.17.layer.1.layer_norm.weight\n",
      "model.encoder.block.18.layer.0.SelfAttention.q.weight model.encoder.block.18.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.18.layer.0.SelfAttention.k.weight model.encoder.block.18.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.18.layer.0.SelfAttention.v.weight model.encoder.block.18.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.18.layer.0.SelfAttention.o.weight model.encoder.block.18.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.18.layer.0.layer_norm.weight model.encoder.block.18.layer.0.layer_norm.weight\n",
      "model.encoder.block.18.layer.1.DenseReluDense.wi_0.weight model.encoder.block.18.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.18.layer.1.DenseReluDense.wi_1.weight model.encoder.block.18.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.18.layer.1.DenseReluDense.wo.weight model.encoder.block.18.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.18.layer.1.layer_norm.weight model.encoder.block.18.layer.1.layer_norm.weight\n",
      "model.encoder.block.19.layer.0.SelfAttention.q.weight model.encoder.block.19.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.19.layer.0.SelfAttention.k.weight model.encoder.block.19.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.19.layer.0.SelfAttention.v.weight model.encoder.block.19.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.19.layer.0.SelfAttention.o.weight model.encoder.block.19.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.19.layer.0.layer_norm.weight model.encoder.block.19.layer.0.layer_norm.weight\n",
      "model.encoder.block.19.layer.1.DenseReluDense.wi_0.weight model.encoder.block.19.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.19.layer.1.DenseReluDense.wi_1.weight model.encoder.block.19.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.19.layer.1.DenseReluDense.wo.weight model.encoder.block.19.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.19.layer.1.layer_norm.weight model.encoder.block.19.layer.1.layer_norm.weight\n",
      "model.encoder.block.20.layer.0.SelfAttention.q.weight model.encoder.block.20.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.20.layer.0.SelfAttention.k.weight model.encoder.block.20.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.20.layer.0.SelfAttention.v.weight model.encoder.block.20.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.20.layer.0.SelfAttention.o.weight model.encoder.block.20.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.20.layer.0.layer_norm.weight model.encoder.block.20.layer.0.layer_norm.weight\n",
      "model.encoder.block.20.layer.1.DenseReluDense.wi_0.weight model.encoder.block.20.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.20.layer.1.DenseReluDense.wi_1.weight model.encoder.block.20.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.20.layer.1.DenseReluDense.wo.weight model.encoder.block.20.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.20.layer.1.layer_norm.weight model.encoder.block.20.layer.1.layer_norm.weight\n",
      "model.encoder.block.21.layer.0.SelfAttention.q.weight model.encoder.block.21.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.21.layer.0.SelfAttention.k.weight model.encoder.block.21.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.21.layer.0.SelfAttention.v.weight model.encoder.block.21.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.21.layer.0.SelfAttention.o.weight model.encoder.block.21.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.21.layer.0.layer_norm.weight model.encoder.block.21.layer.0.layer_norm.weight\n",
      "model.encoder.block.21.layer.1.DenseReluDense.wi_0.weight model.encoder.block.21.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.21.layer.1.DenseReluDense.wi_1.weight model.encoder.block.21.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.21.layer.1.DenseReluDense.wo.weight model.encoder.block.21.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.21.layer.1.layer_norm.weight model.encoder.block.21.layer.1.layer_norm.weight\n",
      "model.encoder.block.22.layer.0.SelfAttention.q.weight model.encoder.block.22.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.22.layer.0.SelfAttention.k.weight model.encoder.block.22.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.22.layer.0.SelfAttention.v.weight model.encoder.block.22.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.22.layer.0.SelfAttention.o.weight model.encoder.block.22.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.22.layer.0.layer_norm.weight model.encoder.block.22.layer.0.layer_norm.weight\n",
      "model.encoder.block.22.layer.1.DenseReluDense.wi_0.weight model.encoder.block.22.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.22.layer.1.DenseReluDense.wi_1.weight model.encoder.block.22.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.22.layer.1.DenseReluDense.wo.weight model.encoder.block.22.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.22.layer.1.layer_norm.weight model.encoder.block.22.layer.1.layer_norm.weight\n",
      "model.encoder.block.23.layer.0.SelfAttention.q.weight model.encoder.block.23.layer.0.SelfAttention.q.weight\n",
      "model.encoder.block.23.layer.0.SelfAttention.k.weight model.encoder.block.23.layer.0.SelfAttention.k.weight\n",
      "model.encoder.block.23.layer.0.SelfAttention.v.weight model.encoder.block.23.layer.0.SelfAttention.v.weight\n",
      "model.encoder.block.23.layer.0.SelfAttention.o.weight model.encoder.block.23.layer.0.SelfAttention.o.weight\n",
      "model.encoder.block.23.layer.0.layer_norm.weight model.encoder.block.23.layer.0.layer_norm.weight\n",
      "model.encoder.block.23.layer.1.DenseReluDense.wi_0.weight model.encoder.block.23.layer.1.DenseReluDense.wi_0.weight\n",
      "model.encoder.block.23.layer.1.DenseReluDense.wi_1.weight model.encoder.block.23.layer.1.DenseReluDense.wi_1.weight\n",
      "model.encoder.block.23.layer.1.DenseReluDense.wo.weight model.encoder.block.23.layer.1.DenseReluDense.wo.weight\n",
      "model.encoder.block.23.layer.1.layer_norm.weight model.encoder.block.23.layer.1.layer_norm.weight\n",
      "model.encoder.final_layer_norm.weight model.encoder.final_layer_norm.weight\n",
      "model.decoder.embed_tokens.weight model.decoder.embed_tokens.weight\n",
      "model.decoder.block.0.layer.0.SelfAttention.q.weight model.decoder.block.0.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.0.layer.0.SelfAttention.k.weight model.decoder.block.0.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.0.layer.0.SelfAttention.v.weight model.decoder.block.0.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.0.layer.0.SelfAttention.o.weight model.decoder.block.0.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight model.decoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight\n",
      "model.decoder.block.0.layer.0.layer_norm.weight model.decoder.block.0.layer.0.layer_norm.weight\n",
      "model.decoder.block.0.layer.1.EncDecAttention.q.weight model.decoder.block.0.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.0.layer.1.EncDecAttention.k.weight model.decoder.block.0.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.0.layer.1.EncDecAttention.v.weight model.decoder.block.0.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.0.layer.1.EncDecAttention.o.weight model.decoder.block.0.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.0.layer.1.layer_norm.weight model.decoder.block.0.layer.1.layer_norm.weight\n",
      "model.decoder.block.0.layer.2.DenseReluDense.wi_0.weight model.decoder.block.0.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.0.layer.2.DenseReluDense.wi_1.weight model.decoder.block.0.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.0.layer.2.DenseReluDense.wo.weight model.decoder.block.0.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.0.layer.2.layer_norm.weight model.decoder.block.0.layer.2.layer_norm.weight\n",
      "model.decoder.block.1.layer.0.SelfAttention.q.weight model.decoder.block.1.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.1.layer.0.SelfAttention.k.weight model.decoder.block.1.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.1.layer.0.SelfAttention.v.weight model.decoder.block.1.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.1.layer.0.SelfAttention.o.weight model.decoder.block.1.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.1.layer.0.layer_norm.weight model.decoder.block.1.layer.0.layer_norm.weight\n",
      "model.decoder.block.1.layer.1.EncDecAttention.q.weight model.decoder.block.1.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.1.layer.1.EncDecAttention.k.weight model.decoder.block.1.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.1.layer.1.EncDecAttention.v.weight model.decoder.block.1.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.1.layer.1.EncDecAttention.o.weight model.decoder.block.1.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.1.layer.1.layer_norm.weight model.decoder.block.1.layer.1.layer_norm.weight\n",
      "model.decoder.block.1.layer.2.DenseReluDense.wi_0.weight model.decoder.block.1.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.1.layer.2.DenseReluDense.wi_1.weight model.decoder.block.1.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.1.layer.2.DenseReluDense.wo.weight model.decoder.block.1.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.1.layer.2.layer_norm.weight model.decoder.block.1.layer.2.layer_norm.weight\n",
      "model.decoder.block.2.layer.0.SelfAttention.q.weight model.decoder.block.2.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.2.layer.0.SelfAttention.k.weight model.decoder.block.2.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.2.layer.0.SelfAttention.v.weight model.decoder.block.2.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.2.layer.0.SelfAttention.o.weight model.decoder.block.2.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.2.layer.0.layer_norm.weight model.decoder.block.2.layer.0.layer_norm.weight\n",
      "model.decoder.block.2.layer.1.EncDecAttention.q.weight model.decoder.block.2.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.2.layer.1.EncDecAttention.k.weight model.decoder.block.2.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.2.layer.1.EncDecAttention.v.weight model.decoder.block.2.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.2.layer.1.EncDecAttention.o.weight model.decoder.block.2.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.2.layer.1.layer_norm.weight model.decoder.block.2.layer.1.layer_norm.weight\n",
      "model.decoder.block.2.layer.2.DenseReluDense.wi_0.weight model.decoder.block.2.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.2.layer.2.DenseReluDense.wi_1.weight model.decoder.block.2.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.2.layer.2.DenseReluDense.wo.weight model.decoder.block.2.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.2.layer.2.layer_norm.weight model.decoder.block.2.layer.2.layer_norm.weight\n",
      "model.decoder.block.3.layer.0.SelfAttention.q.weight model.decoder.block.3.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.3.layer.0.SelfAttention.k.weight model.decoder.block.3.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.3.layer.0.SelfAttention.v.weight model.decoder.block.3.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.3.layer.0.SelfAttention.o.weight model.decoder.block.3.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.3.layer.0.layer_norm.weight model.decoder.block.3.layer.0.layer_norm.weight\n",
      "model.decoder.block.3.layer.1.EncDecAttention.q.weight model.decoder.block.3.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.3.layer.1.EncDecAttention.k.weight model.decoder.block.3.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.3.layer.1.EncDecAttention.v.weight model.decoder.block.3.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.3.layer.1.EncDecAttention.o.weight model.decoder.block.3.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.3.layer.1.layer_norm.weight model.decoder.block.3.layer.1.layer_norm.weight\n",
      "model.decoder.block.3.layer.2.DenseReluDense.wi_0.weight model.decoder.block.3.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.3.layer.2.DenseReluDense.wi_1.weight model.decoder.block.3.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.3.layer.2.DenseReluDense.wo.weight model.decoder.block.3.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.3.layer.2.layer_norm.weight model.decoder.block.3.layer.2.layer_norm.weight\n",
      "model.decoder.block.4.layer.0.SelfAttention.q.weight model.decoder.block.4.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.4.layer.0.SelfAttention.k.weight model.decoder.block.4.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.4.layer.0.SelfAttention.v.weight model.decoder.block.4.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.4.layer.0.SelfAttention.o.weight model.decoder.block.4.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.4.layer.0.layer_norm.weight model.decoder.block.4.layer.0.layer_norm.weight\n",
      "model.decoder.block.4.layer.1.EncDecAttention.q.weight model.decoder.block.4.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.4.layer.1.EncDecAttention.k.weight model.decoder.block.4.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.4.layer.1.EncDecAttention.v.weight model.decoder.block.4.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.4.layer.1.EncDecAttention.o.weight model.decoder.block.4.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.4.layer.1.layer_norm.weight model.decoder.block.4.layer.1.layer_norm.weight\n",
      "model.decoder.block.4.layer.2.DenseReluDense.wi_0.weight model.decoder.block.4.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.4.layer.2.DenseReluDense.wi_1.weight model.decoder.block.4.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.4.layer.2.DenseReluDense.wo.weight model.decoder.block.4.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.4.layer.2.layer_norm.weight model.decoder.block.4.layer.2.layer_norm.weight\n",
      "model.decoder.block.5.layer.0.SelfAttention.q.weight model.decoder.block.5.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.5.layer.0.SelfAttention.k.weight model.decoder.block.5.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.5.layer.0.SelfAttention.v.weight model.decoder.block.5.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.5.layer.0.SelfAttention.o.weight model.decoder.block.5.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.5.layer.0.layer_norm.weight model.decoder.block.5.layer.0.layer_norm.weight\n",
      "model.decoder.block.5.layer.1.EncDecAttention.q.weight model.decoder.block.5.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.5.layer.1.EncDecAttention.k.weight model.decoder.block.5.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.5.layer.1.EncDecAttention.v.weight model.decoder.block.5.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.5.layer.1.EncDecAttention.o.weight model.decoder.block.5.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.5.layer.1.layer_norm.weight model.decoder.block.5.layer.1.layer_norm.weight\n",
      "model.decoder.block.5.layer.2.DenseReluDense.wi_0.weight model.decoder.block.5.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.5.layer.2.DenseReluDense.wi_1.weight model.decoder.block.5.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.5.layer.2.DenseReluDense.wo.weight model.decoder.block.5.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.5.layer.2.layer_norm.weight model.decoder.block.5.layer.2.layer_norm.weight\n",
      "model.decoder.block.6.layer.0.SelfAttention.q.weight model.decoder.block.6.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.6.layer.0.SelfAttention.k.weight model.decoder.block.6.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.6.layer.0.SelfAttention.v.weight model.decoder.block.6.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.6.layer.0.SelfAttention.o.weight model.decoder.block.6.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.6.layer.0.layer_norm.weight model.decoder.block.6.layer.0.layer_norm.weight\n",
      "model.decoder.block.6.layer.1.EncDecAttention.q.weight model.decoder.block.6.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.6.layer.1.EncDecAttention.k.weight model.decoder.block.6.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.6.layer.1.EncDecAttention.v.weight model.decoder.block.6.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.6.layer.1.EncDecAttention.o.weight model.decoder.block.6.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.6.layer.1.layer_norm.weight model.decoder.block.6.layer.1.layer_norm.weight\n",
      "model.decoder.block.6.layer.2.DenseReluDense.wi_0.weight model.decoder.block.6.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.6.layer.2.DenseReluDense.wi_1.weight model.decoder.block.6.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.6.layer.2.DenseReluDense.wo.weight model.decoder.block.6.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.6.layer.2.layer_norm.weight model.decoder.block.6.layer.2.layer_norm.weight\n",
      "model.decoder.block.7.layer.0.SelfAttention.q.weight model.decoder.block.7.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.7.layer.0.SelfAttention.k.weight model.decoder.block.7.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.7.layer.0.SelfAttention.v.weight model.decoder.block.7.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.7.layer.0.SelfAttention.o.weight model.decoder.block.7.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.7.layer.0.layer_norm.weight model.decoder.block.7.layer.0.layer_norm.weight\n",
      "model.decoder.block.7.layer.1.EncDecAttention.q.weight model.decoder.block.7.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.7.layer.1.EncDecAttention.k.weight model.decoder.block.7.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.7.layer.1.EncDecAttention.v.weight model.decoder.block.7.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.7.layer.1.EncDecAttention.o.weight model.decoder.block.7.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.7.layer.1.layer_norm.weight model.decoder.block.7.layer.1.layer_norm.weight\n",
      "model.decoder.block.7.layer.2.DenseReluDense.wi_0.weight model.decoder.block.7.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.7.layer.2.DenseReluDense.wi_1.weight model.decoder.block.7.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.7.layer.2.DenseReluDense.wo.weight model.decoder.block.7.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.7.layer.2.layer_norm.weight model.decoder.block.7.layer.2.layer_norm.weight\n",
      "model.decoder.block.8.layer.0.SelfAttention.q.weight model.decoder.block.8.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.8.layer.0.SelfAttention.k.weight model.decoder.block.8.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.8.layer.0.SelfAttention.v.weight model.decoder.block.8.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.8.layer.0.SelfAttention.o.weight model.decoder.block.8.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.8.layer.0.layer_norm.weight model.decoder.block.8.layer.0.layer_norm.weight\n",
      "model.decoder.block.8.layer.1.EncDecAttention.q.weight model.decoder.block.8.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.8.layer.1.EncDecAttention.k.weight model.decoder.block.8.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.8.layer.1.EncDecAttention.v.weight model.decoder.block.8.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.8.layer.1.EncDecAttention.o.weight model.decoder.block.8.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.8.layer.1.layer_norm.weight model.decoder.block.8.layer.1.layer_norm.weight\n",
      "model.decoder.block.8.layer.2.DenseReluDense.wi_0.weight model.decoder.block.8.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.8.layer.2.DenseReluDense.wi_1.weight model.decoder.block.8.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.8.layer.2.DenseReluDense.wo.weight model.decoder.block.8.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.8.layer.2.layer_norm.weight model.decoder.block.8.layer.2.layer_norm.weight\n",
      "model.decoder.block.9.layer.0.SelfAttention.q.weight model.decoder.block.9.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.9.layer.0.SelfAttention.k.weight model.decoder.block.9.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.9.layer.0.SelfAttention.v.weight model.decoder.block.9.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.9.layer.0.SelfAttention.o.weight model.decoder.block.9.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.9.layer.0.layer_norm.weight model.decoder.block.9.layer.0.layer_norm.weight\n",
      "model.decoder.block.9.layer.1.EncDecAttention.q.weight model.decoder.block.9.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.9.layer.1.EncDecAttention.k.weight model.decoder.block.9.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.9.layer.1.EncDecAttention.v.weight model.decoder.block.9.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.9.layer.1.EncDecAttention.o.weight model.decoder.block.9.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.9.layer.1.layer_norm.weight model.decoder.block.9.layer.1.layer_norm.weight\n",
      "model.decoder.block.9.layer.2.DenseReluDense.wi_0.weight model.decoder.block.9.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.9.layer.2.DenseReluDense.wi_1.weight model.decoder.block.9.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.9.layer.2.DenseReluDense.wo.weight model.decoder.block.9.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.9.layer.2.layer_norm.weight model.decoder.block.9.layer.2.layer_norm.weight\n",
      "model.decoder.block.10.layer.0.SelfAttention.q.weight model.decoder.block.10.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.10.layer.0.SelfAttention.k.weight model.decoder.block.10.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.10.layer.0.SelfAttention.v.weight model.decoder.block.10.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.10.layer.0.SelfAttention.o.weight model.decoder.block.10.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.10.layer.0.layer_norm.weight model.decoder.block.10.layer.0.layer_norm.weight\n",
      "model.decoder.block.10.layer.1.EncDecAttention.q.weight model.decoder.block.10.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.10.layer.1.EncDecAttention.k.weight model.decoder.block.10.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.10.layer.1.EncDecAttention.v.weight model.decoder.block.10.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.10.layer.1.EncDecAttention.o.weight model.decoder.block.10.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.10.layer.1.layer_norm.weight model.decoder.block.10.layer.1.layer_norm.weight\n",
      "model.decoder.block.10.layer.2.DenseReluDense.wi_0.weight model.decoder.block.10.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.10.layer.2.DenseReluDense.wi_1.weight model.decoder.block.10.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.10.layer.2.DenseReluDense.wo.weight model.decoder.block.10.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.10.layer.2.layer_norm.weight model.decoder.block.10.layer.2.layer_norm.weight\n",
      "model.decoder.block.11.layer.0.SelfAttention.q.weight model.decoder.block.11.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.11.layer.0.SelfAttention.k.weight model.decoder.block.11.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.11.layer.0.SelfAttention.v.weight model.decoder.block.11.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.11.layer.0.SelfAttention.o.weight model.decoder.block.11.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.11.layer.0.layer_norm.weight model.decoder.block.11.layer.0.layer_norm.weight\n",
      "model.decoder.block.11.layer.1.EncDecAttention.q.weight model.decoder.block.11.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.11.layer.1.EncDecAttention.k.weight model.decoder.block.11.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.11.layer.1.EncDecAttention.v.weight model.decoder.block.11.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.11.layer.1.EncDecAttention.o.weight model.decoder.block.11.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.11.layer.1.layer_norm.weight model.decoder.block.11.layer.1.layer_norm.weight\n",
      "model.decoder.block.11.layer.2.DenseReluDense.wi_0.weight model.decoder.block.11.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.11.layer.2.DenseReluDense.wi_1.weight model.decoder.block.11.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.11.layer.2.DenseReluDense.wo.weight model.decoder.block.11.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.11.layer.2.layer_norm.weight model.decoder.block.11.layer.2.layer_norm.weight\n",
      "model.decoder.block.12.layer.0.SelfAttention.q.weight model.decoder.block.12.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.12.layer.0.SelfAttention.k.weight model.decoder.block.12.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.12.layer.0.SelfAttention.v.weight model.decoder.block.12.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.12.layer.0.SelfAttention.o.weight model.decoder.block.12.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.12.layer.0.layer_norm.weight model.decoder.block.12.layer.0.layer_norm.weight\n",
      "model.decoder.block.12.layer.1.EncDecAttention.q.weight model.decoder.block.12.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.12.layer.1.EncDecAttention.k.weight model.decoder.block.12.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.12.layer.1.EncDecAttention.v.weight model.decoder.block.12.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.12.layer.1.EncDecAttention.o.weight model.decoder.block.12.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.12.layer.1.layer_norm.weight model.decoder.block.12.layer.1.layer_norm.weight\n",
      "model.decoder.block.12.layer.2.DenseReluDense.wi_0.weight model.decoder.block.12.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.12.layer.2.DenseReluDense.wi_1.weight model.decoder.block.12.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.12.layer.2.DenseReluDense.wo.weight model.decoder.block.12.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.12.layer.2.layer_norm.weight model.decoder.block.12.layer.2.layer_norm.weight\n",
      "model.decoder.block.13.layer.0.SelfAttention.q.weight model.decoder.block.13.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.13.layer.0.SelfAttention.k.weight model.decoder.block.13.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.13.layer.0.SelfAttention.v.weight model.decoder.block.13.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.13.layer.0.SelfAttention.o.weight model.decoder.block.13.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.13.layer.0.layer_norm.weight model.decoder.block.13.layer.0.layer_norm.weight\n",
      "model.decoder.block.13.layer.1.EncDecAttention.q.weight model.decoder.block.13.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.13.layer.1.EncDecAttention.k.weight model.decoder.block.13.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.13.layer.1.EncDecAttention.v.weight model.decoder.block.13.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.13.layer.1.EncDecAttention.o.weight model.decoder.block.13.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.13.layer.1.layer_norm.weight model.decoder.block.13.layer.1.layer_norm.weight\n",
      "model.decoder.block.13.layer.2.DenseReluDense.wi_0.weight model.decoder.block.13.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.13.layer.2.DenseReluDense.wi_1.weight model.decoder.block.13.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.13.layer.2.DenseReluDense.wo.weight model.decoder.block.13.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.13.layer.2.layer_norm.weight model.decoder.block.13.layer.2.layer_norm.weight\n",
      "model.decoder.block.14.layer.0.SelfAttention.q.weight model.decoder.block.14.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.14.layer.0.SelfAttention.k.weight model.decoder.block.14.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.14.layer.0.SelfAttention.v.weight model.decoder.block.14.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.14.layer.0.SelfAttention.o.weight model.decoder.block.14.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.14.layer.0.layer_norm.weight model.decoder.block.14.layer.0.layer_norm.weight\n",
      "model.decoder.block.14.layer.1.EncDecAttention.q.weight model.decoder.block.14.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.14.layer.1.EncDecAttention.k.weight model.decoder.block.14.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.14.layer.1.EncDecAttention.v.weight model.decoder.block.14.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.14.layer.1.EncDecAttention.o.weight model.decoder.block.14.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.14.layer.1.layer_norm.weight model.decoder.block.14.layer.1.layer_norm.weight\n",
      "model.decoder.block.14.layer.2.DenseReluDense.wi_0.weight model.decoder.block.14.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.14.layer.2.DenseReluDense.wi_1.weight model.decoder.block.14.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.14.layer.2.DenseReluDense.wo.weight model.decoder.block.14.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.14.layer.2.layer_norm.weight model.decoder.block.14.layer.2.layer_norm.weight\n",
      "model.decoder.block.15.layer.0.SelfAttention.q.weight model.decoder.block.15.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.15.layer.0.SelfAttention.k.weight model.decoder.block.15.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.15.layer.0.SelfAttention.v.weight model.decoder.block.15.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.15.layer.0.SelfAttention.o.weight model.decoder.block.15.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.15.layer.0.layer_norm.weight model.decoder.block.15.layer.0.layer_norm.weight\n",
      "model.decoder.block.15.layer.1.EncDecAttention.q.weight model.decoder.block.15.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.15.layer.1.EncDecAttention.k.weight model.decoder.block.15.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.15.layer.1.EncDecAttention.v.weight model.decoder.block.15.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.15.layer.1.EncDecAttention.o.weight model.decoder.block.15.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.15.layer.1.layer_norm.weight model.decoder.block.15.layer.1.layer_norm.weight\n",
      "model.decoder.block.15.layer.2.DenseReluDense.wi_0.weight model.decoder.block.15.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.15.layer.2.DenseReluDense.wi_1.weight model.decoder.block.15.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.15.layer.2.DenseReluDense.wo.weight model.decoder.block.15.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.15.layer.2.layer_norm.weight model.decoder.block.15.layer.2.layer_norm.weight\n",
      "model.decoder.block.16.layer.0.SelfAttention.q.weight model.decoder.block.16.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.16.layer.0.SelfAttention.k.weight model.decoder.block.16.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.16.layer.0.SelfAttention.v.weight model.decoder.block.16.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.16.layer.0.SelfAttention.o.weight model.decoder.block.16.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.16.layer.0.layer_norm.weight model.decoder.block.16.layer.0.layer_norm.weight\n",
      "model.decoder.block.16.layer.1.EncDecAttention.q.weight model.decoder.block.16.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.16.layer.1.EncDecAttention.k.weight model.decoder.block.16.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.16.layer.1.EncDecAttention.v.weight model.decoder.block.16.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.16.layer.1.EncDecAttention.o.weight model.decoder.block.16.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.16.layer.1.layer_norm.weight model.decoder.block.16.layer.1.layer_norm.weight\n",
      "model.decoder.block.16.layer.2.DenseReluDense.wi_0.weight model.decoder.block.16.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.16.layer.2.DenseReluDense.wi_1.weight model.decoder.block.16.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.16.layer.2.DenseReluDense.wo.weight model.decoder.block.16.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.16.layer.2.layer_norm.weight model.decoder.block.16.layer.2.layer_norm.weight\n",
      "model.decoder.block.17.layer.0.SelfAttention.q.weight model.decoder.block.17.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.17.layer.0.SelfAttention.k.weight model.decoder.block.17.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.17.layer.0.SelfAttention.v.weight model.decoder.block.17.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.17.layer.0.SelfAttention.o.weight model.decoder.block.17.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.17.layer.0.layer_norm.weight model.decoder.block.17.layer.0.layer_norm.weight\n",
      "model.decoder.block.17.layer.1.EncDecAttention.q.weight model.decoder.block.17.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.17.layer.1.EncDecAttention.k.weight model.decoder.block.17.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.17.layer.1.EncDecAttention.v.weight model.decoder.block.17.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.17.layer.1.EncDecAttention.o.weight model.decoder.block.17.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.17.layer.1.layer_norm.weight model.decoder.block.17.layer.1.layer_norm.weight\n",
      "model.decoder.block.17.layer.2.DenseReluDense.wi_0.weight model.decoder.block.17.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.17.layer.2.DenseReluDense.wi_1.weight model.decoder.block.17.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.17.layer.2.DenseReluDense.wo.weight model.decoder.block.17.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.17.layer.2.layer_norm.weight model.decoder.block.17.layer.2.layer_norm.weight\n",
      "model.decoder.block.18.layer.0.SelfAttention.q.weight model.decoder.block.18.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.18.layer.0.SelfAttention.k.weight model.decoder.block.18.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.18.layer.0.SelfAttention.v.weight model.decoder.block.18.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.18.layer.0.SelfAttention.o.weight model.decoder.block.18.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.18.layer.0.layer_norm.weight model.decoder.block.18.layer.0.layer_norm.weight\n",
      "model.decoder.block.18.layer.1.EncDecAttention.q.weight model.decoder.block.18.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.18.layer.1.EncDecAttention.k.weight model.decoder.block.18.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.18.layer.1.EncDecAttention.v.weight model.decoder.block.18.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.18.layer.1.EncDecAttention.o.weight model.decoder.block.18.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.18.layer.1.layer_norm.weight model.decoder.block.18.layer.1.layer_norm.weight\n",
      "model.decoder.block.18.layer.2.DenseReluDense.wi_0.weight model.decoder.block.18.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.18.layer.2.DenseReluDense.wi_1.weight model.decoder.block.18.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.18.layer.2.DenseReluDense.wo.weight model.decoder.block.18.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.18.layer.2.layer_norm.weight model.decoder.block.18.layer.2.layer_norm.weight\n",
      "model.decoder.block.19.layer.0.SelfAttention.q.weight model.decoder.block.19.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.19.layer.0.SelfAttention.k.weight model.decoder.block.19.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.19.layer.0.SelfAttention.v.weight model.decoder.block.19.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.19.layer.0.SelfAttention.o.weight model.decoder.block.19.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.19.layer.0.layer_norm.weight model.decoder.block.19.layer.0.layer_norm.weight\n",
      "model.decoder.block.19.layer.1.EncDecAttention.q.weight model.decoder.block.19.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.19.layer.1.EncDecAttention.k.weight model.decoder.block.19.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.19.layer.1.EncDecAttention.v.weight model.decoder.block.19.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.19.layer.1.EncDecAttention.o.weight model.decoder.block.19.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.19.layer.1.layer_norm.weight model.decoder.block.19.layer.1.layer_norm.weight\n",
      "model.decoder.block.19.layer.2.DenseReluDense.wi_0.weight model.decoder.block.19.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.19.layer.2.DenseReluDense.wi_1.weight model.decoder.block.19.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.19.layer.2.DenseReluDense.wo.weight model.decoder.block.19.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.19.layer.2.layer_norm.weight model.decoder.block.19.layer.2.layer_norm.weight\n",
      "model.decoder.block.20.layer.0.SelfAttention.q.weight model.decoder.block.20.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.20.layer.0.SelfAttention.k.weight model.decoder.block.20.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.20.layer.0.SelfAttention.v.weight model.decoder.block.20.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.20.layer.0.SelfAttention.o.weight model.decoder.block.20.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.20.layer.0.layer_norm.weight model.decoder.block.20.layer.0.layer_norm.weight\n",
      "model.decoder.block.20.layer.1.EncDecAttention.q.weight model.decoder.block.20.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.20.layer.1.EncDecAttention.k.weight model.decoder.block.20.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.20.layer.1.EncDecAttention.v.weight model.decoder.block.20.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.20.layer.1.EncDecAttention.o.weight model.decoder.block.20.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.20.layer.1.layer_norm.weight model.decoder.block.20.layer.1.layer_norm.weight\n",
      "model.decoder.block.20.layer.2.DenseReluDense.wi_0.weight model.decoder.block.20.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.20.layer.2.DenseReluDense.wi_1.weight model.decoder.block.20.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.20.layer.2.DenseReluDense.wo.weight model.decoder.block.20.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.20.layer.2.layer_norm.weight model.decoder.block.20.layer.2.layer_norm.weight\n",
      "model.decoder.block.21.layer.0.SelfAttention.q.weight model.decoder.block.21.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.21.layer.0.SelfAttention.k.weight model.decoder.block.21.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.21.layer.0.SelfAttention.v.weight model.decoder.block.21.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.21.layer.0.SelfAttention.o.weight model.decoder.block.21.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.21.layer.0.layer_norm.weight model.decoder.block.21.layer.0.layer_norm.weight\n",
      "model.decoder.block.21.layer.1.EncDecAttention.q.weight model.decoder.block.21.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.21.layer.1.EncDecAttention.k.weight model.decoder.block.21.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.21.layer.1.EncDecAttention.v.weight model.decoder.block.21.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.21.layer.1.EncDecAttention.o.weight model.decoder.block.21.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.21.layer.1.layer_norm.weight model.decoder.block.21.layer.1.layer_norm.weight\n",
      "model.decoder.block.21.layer.2.DenseReluDense.wi_0.weight model.decoder.block.21.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.21.layer.2.DenseReluDense.wi_1.weight model.decoder.block.21.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.21.layer.2.DenseReluDense.wo.weight model.decoder.block.21.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.21.layer.2.layer_norm.weight model.decoder.block.21.layer.2.layer_norm.weight\n",
      "model.decoder.block.22.layer.0.SelfAttention.q.weight model.decoder.block.22.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.22.layer.0.SelfAttention.k.weight model.decoder.block.22.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.22.layer.0.SelfAttention.v.weight model.decoder.block.22.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.22.layer.0.SelfAttention.o.weight model.decoder.block.22.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.22.layer.0.layer_norm.weight model.decoder.block.22.layer.0.layer_norm.weight\n",
      "model.decoder.block.22.layer.1.EncDecAttention.q.weight model.decoder.block.22.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.22.layer.1.EncDecAttention.k.weight model.decoder.block.22.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.22.layer.1.EncDecAttention.v.weight model.decoder.block.22.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.22.layer.1.EncDecAttention.o.weight model.decoder.block.22.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.22.layer.1.layer_norm.weight model.decoder.block.22.layer.1.layer_norm.weight\n",
      "model.decoder.block.22.layer.2.DenseReluDense.wi_0.weight model.decoder.block.22.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.22.layer.2.DenseReluDense.wi_1.weight model.decoder.block.22.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.22.layer.2.DenseReluDense.wo.weight model.decoder.block.22.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.22.layer.2.layer_norm.weight model.decoder.block.22.layer.2.layer_norm.weight\n",
      "model.decoder.block.23.layer.0.SelfAttention.q.weight model.decoder.block.23.layer.0.SelfAttention.q.weight\n",
      "model.decoder.block.23.layer.0.SelfAttention.k.weight model.decoder.block.23.layer.0.SelfAttention.k.weight\n",
      "model.decoder.block.23.layer.0.SelfAttention.v.weight model.decoder.block.23.layer.0.SelfAttention.v.weight\n",
      "model.decoder.block.23.layer.0.SelfAttention.o.weight model.decoder.block.23.layer.0.SelfAttention.o.weight\n",
      "model.decoder.block.23.layer.0.layer_norm.weight model.decoder.block.23.layer.0.layer_norm.weight\n",
      "model.decoder.block.23.layer.1.EncDecAttention.q.weight model.decoder.block.23.layer.1.EncDecAttention.q.weight\n",
      "model.decoder.block.23.layer.1.EncDecAttention.k.weight model.decoder.block.23.layer.1.EncDecAttention.k.weight\n",
      "model.decoder.block.23.layer.1.EncDecAttention.v.weight model.decoder.block.23.layer.1.EncDecAttention.v.weight\n",
      "model.decoder.block.23.layer.1.EncDecAttention.o.weight model.decoder.block.23.layer.1.EncDecAttention.o.weight\n",
      "model.decoder.block.23.layer.1.layer_norm.weight model.decoder.block.23.layer.1.layer_norm.weight\n",
      "model.decoder.block.23.layer.2.DenseReluDense.wi_0.weight model.decoder.block.23.layer.2.DenseReluDense.wi_0.weight\n",
      "model.decoder.block.23.layer.2.DenseReluDense.wi_1.weight model.decoder.block.23.layer.2.DenseReluDense.wi_1.weight\n",
      "model.decoder.block.23.layer.2.DenseReluDense.wo.weight model.decoder.block.23.layer.2.DenseReluDense.wo.weight\n",
      "model.decoder.block.23.layer.2.layer_norm.weight model.decoder.block.23.layer.2.layer_norm.weight\n",
      "model.decoder.final_layer_norm.weight model.decoder.final_layer_norm.weight\n",
      "model.lm_head.weight model.lm_head.weight\n"
     ]
    }
   ],
   "source": [
    "for k, v in old_weights:\n",
    "    new_k = k.replace('._orig_mod', '')\n",
    "    print(k, new_k)\n",
    "    weights[new_k] = v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "839ae22f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c698e532",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.copied_utils import (\n",
    "    compute_input_and_target_lengths,\n",
    "    DataCollatorForT5MLM,\n",
    "    tokenize_function,\n",
    "    DataCollatorForNI,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "60e223c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(568, 114)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "before_mask_input_length, target_length = compute_input_and_target_lengths(\n",
    "    inputs_length=512,\n",
    "    noise_density=0.15,\n",
    "    mean_noise_span_length=3.0,\n",
    ")\n",
    "before_mask_input_length, target_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e78b0fbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from streaming.base.format.mds.encodings import Encoding, _encodings\n",
    "from streaming import StreamingDataset\n",
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "class Int32(Encoding):\n",
    "    def encode(self, obj) -> bytes:\n",
    "        return obj.tobytes()\n",
    "\n",
    "    def decode(self, data: bytes):\n",
    "        return np.frombuffer(data, np.int32)\n",
    "\n",
    "\n",
    "_encodings['int32'] = Int32\n",
    "\n",
    "\n",
    "class DatasetFixed(torch.utils.data.Dataset):\n",
    "    def __init__(self, local):\n",
    "        self.dataset = StreamingDataset(local=local)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        data = self.dataset[idx]\n",
    "        data.pop('token_type_ids', None)\n",
    "        for k in data.keys():\n",
    "            data[k] = data[k].astype(np.int64)\n",
    "        return data\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d3c2bef2",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = DatasetFixed(local='/home/ubuntu/nanot5-512')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "317b0cda",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorForT5MLM(\n",
    "    tokenizer=tokenizer,\n",
    "    noise_density=0.15,\n",
    "    mean_noise_span_length=3.0,\n",
    "    input_length=512,\n",
    "    target_length=target_length,\n",
    "    pad_token_id=0,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4028b237",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = [dataset[i] for i in range(5000000, 5000001, 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d788fba7",
   "metadata": {},
   "outputs": [],
   "source": [
    "padded = data_collator(batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ad3a2d8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "o = model.model(**padded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ba335b10",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-2.7402,  2.5614,  6.1874,  ..., -1.3941, -3.1445, -4.2745],\n",
       "         [-3.2996,  1.7034,  7.6094,  ..., -0.9311, -2.4005, -2.0222],\n",
       "         [-2.5788,  1.4604,  7.6094,  ..., -0.8557, -1.9514, -1.8257],\n",
       "         ...,\n",
       "         [-1.7656,  2.4665,  6.3454,  ..., -1.3973, -0.4826, -1.8745],\n",
       "         [-1.3759,  2.9568,  7.9547,  ..., -1.9363, -2.1684, -3.5167],\n",
       "         [-1.0695,  3.1586, 28.7467,  ..., -2.2941, -2.2790, -4.0657]]],\n",
       "       grad_fn=<UnsafeViewBackward0>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "o.logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e29488ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<extra_id_40> kilang Swarovs<extra_id_59> seorang husband reseller tu,<extra_id_38> roma<extra_id_18> Bini makin<extra_id_46>-2022 06:28 PM Batu api sungg<extra_id_66>iden gil<extra_id_71> seseorang yang amat bermakna dalam<extra_id_67> menerima<extra_id_6> hidup,<extra_id_53> penulis<extra_id_10>wah persahabatan atau<extra_id_61> agar<extra_id_2>tah<extra_id_84> tiada jawapan<extra_id_8> Membuka pintu<extra_id_28> yang<extra_id_70> Bloom<extra_id_76>z?<extra_id_62>...tah br<extra_id_54> duit...huhuh...<extra_id_12> banyak kali ktorang putus...<extra_id_0>s<extra_id_75> nk tgu.....<extra_id_85> tapi duit<extra_id_83>gu<extra_id_52> duit<extra_id_89>....pac<extra_id_77>...mkn la</s>'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "padded['labels']\n",
    "tokenizer.decode(padded['labels'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3206b0ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<extra_id_40> B<extra_id_59><extra_id_59><extra_id_59><extra_id_59><extra_id_59>. lelaki.it..<extra_id_38>.atu<extra_id_18> akutw<extra_id_46><extra_id_46>-2022 10:41 PM S C tu<extra_id_66><extra_id_66>em<extra_id_71><extra_id_71><extra_id_71> dan<extra_id_67> tidak<extra_id_67><extra_id_67><extra_id_67> tidak<extra_id_6> tidak<extra_id_53><extra_id_53>nya<extra_id_10>awan.is<extra_id_61><extra_id_61>.<extra_id_2>ak<extra_id_84> tidak<extra_id_8> yang Berikan hati<extra_id_28> yang<extra_id_70>,twberg.<extra_id_62><extra_id_62>...<extra_id_54>2<extra_id_54>......<extra_id_12>uh<extra_id_12><extra_id_12><extra_id_12><extra_id_0>2... die......<extra_id_0>s<extra_id_75> b t<extra_id_85>...<extra_id_85>...<extra_id_83><extra_id_83>gk<extra_id_52> aku<extra_id_89>k</s><extra_id_77><extra_id_77>...</s>cm aku</s>'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(o.logits.argmax(-1).detach().cpu().numpy()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14cbcce3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
